/*
 * Copyright (c) Huawei Technologies Co., Ltd. 2020-2022. All rights reserved.
 * Description: hccl实现基类头文件
 * Author: hanwei
 * Create: 2020-01-05
 */

#ifndef HCCL_IMPL_BASE_H
#define HCCL_IMPL_BASE_H

#include <atomic>
#include <memory>
#include <hccl/hccl_types.h>

#include "hccl/base.h"
#include "hccl_impl_pub.h"
#include "executor_impl.h"
#include "hccl_comm_pub.h"
#include "threadManage.h"
#include "network_manager_pub.h"
#include "tcp_send_thread_pool.h"
#include "tcp_recv_task.h"

#include "op_remote_look_up_executor.h"
#include "op_coll_remote_update_executor.h"
#include "alg_base.h"
#include "comm_factory_pub.h"
#include "opexecounter_pub.h"
#include "parallel_task_loader.h"
#include "spin_mutex.h"

#ifndef HCCD
#include "tbe_unsorted_segment_sum_aicore.h"
#include "tbe_crack_cleared.h"
#endif

#include "coll_alg_utils.h"
#include "profiler_manager.h"

#include "topoinfo_parse.h"
#include "hccl_alg.h"
#include "ccl_buffer_manager.h"
#include "hccl_trace_info.h"
#include "hccl_callback_task.h"
#include "atrace_api.h"
#include "hashtable/universal_concurrent_map.h"
#include "resource_manager/queue_notify_manager.h"
#include "workflow_pub.h"
#include "hccl_network_pub.h"

namespace hccl {
using ServRankInfo_t = std::map<std::string, std::vector<RankInfo_t> >;

constexpr u32 OP_BASE_KEY_TRACE_INFO_MAX_SIZE = 100; // 保存opbase层关键日志信息条目个数

constexpr s32 MAX_DEVICE_ID = 31;
constexpr u32 MAX_WORK_SPACE_SIZE = 16 * 1024 * 1024; // 默认16MB
const std::string LOOPBACK_IP = "127.0.0.1";

using TransportStorageMap =
    std::unordered_map<TransportEndPointInfo, std::unique_ptr<TransportHeterog>, TransportEndPointInfoHash>;

using HcclOpCommDesc = struct HcclOpCommDescDef {
    void* inputPtr;
    u64 count;
    HcclDataType dataType;
    u32 peerRank;
    u32 status;
    HcclOpCommDescDef (void *inputPtr, u64 count, HcclDataType dataType, u32 peerRank)
        : inputPtr(inputPtr), count(count), dataType(dataType), peerRank(peerRank), status(0xFFFFFFFF)
    {
    }
};

using HcclOpBaseKeyTraceInfoDesc = struct HcclOpBaseKeyTraceInfoDef {
    struct timeval tv;
    s32 tid;
    std::string keyTraceInfo;
    u32 index; // 同一个通信域内的算子下发count计数
    HcclOpBaseKeyTraceInfoDef() {};
    HcclOpBaseKeyTraceInfoDef(struct timeval tv, s32 tid, std::string keyTraceInfo, u32 index)
        : tv(tv), tid(tid), keyTraceInfo(keyTraceInfo), index(index)
    {
    }
};

class HcclImplBase {
public:
    static std::string GetUniqueId(void);

public:
    explicit HcclImplBase();

    virtual ~HcclImplBase();

    // 对外接口
    virtual HcclResult Init(HcclCommParams &params, const RankTable_t &rankTable);
    virtual HcclResult Init(HcclCommParams &params, const std::vector<RankInfo> &rankList,
        WorldGroupInfo &globalData);

    virtual HcclResult GetAlgType(AlgType &algType, HcclCMDType opType);

    virtual HcclResult GetDeviceNumPerAggregation(u32 &deviceNumPerAggregation);

    virtual HcclResult GetBandWidthPerNPU(u32 level, float &bandWidth);

    u32 GetRankTableCrc();

    virtual HcclResult CreateOrUpdateTransferMem(HcclOpPtr opPtr);
    // 为Embedding Service算子注册workspace资源
    virtual HcclResult RegEsWorkSpaceResource(HcclOpPtr opPtr);

    virtual HcclResult ServiceCancel(HcclOpPtr opPtr);

    virtual HcclResult SupportDeterministicOptim(bool &isDeterministicOptim);

    //  对内接口
    virtual HcclResult CheckDataType(const HcclDataType dataType, bool needReduce);

    virtual HcclResult CheckReduceDataType(const HcclDataType dataType, const HcclReduceOp op);

    virtual HcclResult ReleaseCommInfos();

    virtual bool IsStandardCard();

    virtual HcclResult RegisterMemory(void* buffer, uint64_t size);

    virtual HcclResult UnregisterMemory(void* buffer);

    HcclResult RegEsPsTransferMem(const HcclOperator<EmbeddingServiceParam> &op);

    HcclResult CheckDeviceType(const DevType deviceType) const;

    HcclResult CheckReductionOp(const HcclReduceOp op) const;

    HcclResult CheckUserRank(const u32 userRank) const;

    HcclResult CheckCount(const u64 count) const;

    HcclResult GetGroupCommonData(WorldGroupInfo &groupCommonData) const;

    HcclResult GetHccsLinkNum(u32 &numHccsLink);

    HcclResult GetGroupRanksInfo(const std::vector<u32> &groupRanks, std::vector<RankInfo> &ranksInfo);

    static bool CompareWithDevicePhyId(const RankInfo_t &left, const RankInfo_t &right);

    static bool CompareWithServerId(const ServerInfo_t &left, const ServerInfo_t &right);

    static bool CompareWithUserRank(const RankInfo &left, const RankInfo &right);

    static bool CompareWithNicName(const NetworkInfo_t &left, const NetworkInfo_t &right);
    u32 GetUserRank();
    u32 GetGroupRank();
    u32 GetRankSize();
    HcclResult GetNicInfo(const NICDeployment &nicDeploy, const u32 curRankIndex,
        const std::vector<RankInfo_t> &servRankList, RankInfo &rankInfo) const;
    /* * 以下两函数用于防止重复初始化 */
    HcclResult AtomicInitSet();
    void AtomicInitClear();
    bool GetNicInitialized();
    HcclResult DestroyNetworkResources();
    HcclResult DisablePreResource();
    HcclResult GetWorkspaceSubStreamNum(u64 &streamNum, u64 dataSize, HcclCMDType opType);
    HcclResult GetWorkspaceMemSize(const std::string &opType, u64 count, HcclDataType dataType,
                                   u32 &rankSize, u64 &size, DevType &deviceType) const;
    HcclResult SetWorkspaceResource(const std::string &tag, void *memPtr, u64 &maxSize,
        std::vector<rtStream_t> &stream);
    HcclResult CreateOpBasedResources(const HcclCMDType &opType, const std::string &tag,
        const HcomCollOpInfo &opInfo);
    HcclResult CreateRemoteOpBasedResources(u64 memSize, const std::string &tag);
    HcclResult CreateOrUpdateRemoteOpBasedResources(u64 memSize, const std::string &tag);
    HcclResult DestroyRemoteOpBasedMem(const std::string &tag);
    void DestroyWorkspaceResource(const std::string &tag);
    HcclResult GetInCCLbuffer(void* &buffer, u64 &size);
    HcclResult GetOutCCLbuffer(void* &buffer, u64 &size);
    void ReleaseCommCCLbuffer();
    HcclResult CreateCommCCLbuffer();
    HcclResult InitCCLbuffer(u64 inCCLbufferSize, u64 outCCLbufferSize);
    HcclResult SetQosCfg(const u32 qosCfg);
    HcclResult ResetQosCfg();
    HcclResult GetQosCfg(u32& qosCfg);

    // 目前支持按tag对资源释放、解绑定
    virtual HcclResult ClearOpResource(const std::string &tag);

    HcclResult SetGlobalWorkSpace(std::vector<void *> &globalWorkSpaceAddr);
    // 获得rdma with reduce算子溢出的task信息后清除
    HcclResult GetandClearOverFlowTasks(std::vector<HcclDumpInfo> &hcclDumpInfo);

    HcclResult GetDeviceId(s32 &deviceId) const;
    HcclResult CsInit(const RoleTableInfo &roleTableInfo);
    HcclResult GetWorkerList(std::vector<u32> &nodeList);
    HcclResult GetTagWorkerList(u32 tag, std::vector<u32> &nodeList);
    HcclResult GetPsMap(std::map<u32, u32> &nodeMap);

    HcclResult RegisterShareMemory();
    HcclResult SetMeshAggregationRankSize(u32 size);

    HcclResult InsertRemoteOpStream(const std::string &tag, std::vector<Stream> &stream);
    virtual void Break();
    HcclResult SetDevicePid(s32 devicePid);

    HcclResult CreateTagAndBuildLinks(int32_t tag, uint32_t workerRanks[], uint64_t workerNum);
    HcclResult DestroyCDomainResource(s32 tag);

    std::unordered_map<std::string, std::map<u32, HcclIpAddress>>& GetPhyIdNicInfo();
    std::vector<u32> GetRanksPort();
    std::vector<RankInfo> GetRanksList();
    HcclResult SetWorldGroupInfo(
        std::unordered_map<std::string, std::map<u32, HcclIpAddress>> phyIdNicInfoMap,
        std::vector<RankInfo> worldRankInfoList, std::vector<u32> ranksPort);
    virtual HcclResult SaveTraceInfo(std::string &logInfo);
    HcclResult GetTopoDesc(HcclTopoDescs *topoDescs, uint32_t topoSize);
    void PrepareCommAsync();

protected:
    bool IsDiffDeviceModule(const std::vector<RankInfo_t> &rankList) const;
    bool GetBoardVersion(std::vector<RoleTableNodeInfo> &nodes_);
    HcclResult InitProfiling();
    HcclResult DeinitProfiling();
    HcclResult InitESProfiler();
    HcclResult InitProfiler();
    HcclResult RegistTaskExceptionHandler() const;
    HcclResult UnRegistTaskExceptionHandler() const;
    bool IsOverFlowInfnanMode() const;
    HcclResult InitCommParams(HcclCommParams &params);
    HcclResult InitPreResource(const RankTable_t &rankTable);
    HcclResult InitTcpMode(const RankTable_t &rankTable) const;
    HcclResult InitRaResource();
    bool IsNeedNicInit();
    HcclResult InitNic();
    HcclResult DeinitNic();
    HcclResult InitHeterogHostNic(void);
    HcclResult DeinitHeterogHostNic(void);
    HcclResult InitHeterogRaResource(const RankTable_t &rankTable);
    HcclResult DeinitHeterogRaResource();
    void DestroyHeterogTransport();

    HcclResult BuildEsWorkerTransportLinks(const s32 &tag);
    HcclResult RegSubStreamAndNotify(HcclOpPtr opPtr, s32 cubeSize);
    HcclResult InitPsRdmaHandle(u32 devLogicId, s32 devicePhyId, const HcclIpAddress &ipAddr,
        bool disabledLiteThread = false, bool enable910ALite = false);
    HcclResult InitHdcPsRdmaHandle();
    HcclResult InitPsRdmaHandle();
    HcclResult BuildShmTransport(const u32 &hostPort, const s32 &tag, const RoleTableNodeInfo& node,
        MachinePara& machinePara, std::unique_ptr<TransportHeterog>& transportPtr);
    HcclResult BuildTcpTransport(const u32 &nodePort, const s32 &tag, const RoleTableNodeInfo& node,
        std::unique_ptr<TransportHeterog>& transportPtr);
    HcclResult BuildRdmaTransport(const u32 &nodePort, const RoleTableNodeInfo& node, MachinePara& machinePara,
        std::unique_ptr<TransportHeterog>& transportPtr);
    HcclResult BuildTransport(const u32 &nodePort, const u32 &hostPort, const s32 &tag,
        const std::vector<RoleTableNodeInfo> &nodesInfo);
    HcclResult CsDeInit();
    HcclResult InitPsRaResource(u32 devLogicId, s32 devicePhyId, const HcclIpAddress &ipAddr,
        u32 port, bool isHostUseDevNic, bool isBoardVersion = false);
    HcclResult InitPsHdcRaResource();
    HcclResult InitPsRaResource();
    HcclResult InitWorkerRaResource();
    HcclResult DeinitPsRaResource(u32 devLogicId, s32 devicePhyId, const HcclIpAddress &ipAddr, u32 port);
    HcclResult DeinitHdcPsRaResource();
    HcclResult DeinitPsRaResource();
    HcclResult DeinitWorkerRaResource();
    HcclResult SetDevice(bool isUseDevMap = false);
    HcclResult ResetDevice();
    HcclResult MrManagerInit();
    HcclResult MrManagerDeInit();
    HcclResult InitRecvMsgAndRequestBuffer();
    HcclResult DeInitTransportMem();
    HcclResult InitMemBlocksAndRecvWrMem();
    HcclResult CreateSrq();
    HcclResult DestroySrq();
    HcclResult SetDeviceThread(bool isUseDevMap);
    HcclResult resetDeviceThread();
    HcclResult CreateCtxAndSetDev(u32 devicePhyId);
    HcclResult CreateAndSaveTransport(MachinePara &machinePara, const u32 &nodePort, const u32 &hostPort,
        const s32 &tag, const RoleTableNodeInfo& node, bool isServerInner);
    HcclResult BuildEsInterTransport(MachinePara &machinePara, const u32 &nodePort,
        const s32 &tag, const RoleTableNodeInfo& node, std::unique_ptr<TransportHeterog> &transportPtr);
    HcclResult BuildEsInnerTransport(MachinePara &machinePara, const u32 &hostPort,
        const s32 &tag, const RoleTableNodeInfo& node, std::unique_ptr<TransportHeterog> &transportPtr);
    HcclResult BuildEsPsTransportLinks(const s32 &tag, bool isTagAssociate);

    HcclResult TransportInitThread(std::unique_ptr<TransportHeterog> &transportPtr, HcclResult &ret);
    HcclResult PrintOpbaseKeyTraceInfo(void);
    HcclResult GetServerNum(const std::vector<RankInfo_t> &ranks);
    u32 GetSuperPodNum(const std::vector<RankInfo> &ranks) const;
    HcclResult GetServerId(const RankTable_t &rankTable);
    HcclResult GetInnerServerAverageDevice(const RankTable_t &rankTable);
    HcclResult GetInnerServerAverageDevice(const std::vector<RankInfo> &rankList);
    HcclResult TransformRankInfoByServerId(const std::vector<RankInfo_t> &rankList,
        ServRankInfo_t &servRankInfo) const;
    HcclResult TransformRankList(const std::vector<RankInfo> &rankListIn,
        std::vector<RankInfo_t> &rankListOut);
    HcclResult CheckDevPhyId(const s32 &devicePhyId) const;
    HcclResult CheckRankTable(const RankTable_t &rankTable, const ServRankInfo_t &servRankInfo);
    HcclResult CheckDevCount(const u32 devNum);
    HcclResult SortRankInfoList();
    HcclResult CheckNicDeploy(NICDeployment nicDeploy, DevType deviceType) const;
    bool Check2N(u32 num) const;
    void CollectAlgoAttr(HcclAlgoAttr &algoAttr);
    void CollectTopoAttr(HcclTopoAttr &topoAttr);
    HcclResult InitPara(const std::string &colectiveId);
    HcclResult GetRankInfoList(const RankTable_t &rankTable);
    u32 CalMeshAggRankSize(int halfDevNum) const;
    HcclResult CalAndSetMeshAggRankSize();
    HcclResult InitHeterogRecvExecutor() const;
    HcclResult InitHdcWhlistAddImpl(const s32 deviceLogicId, const HcclIpAddress &ipAddr,
        std::vector<socket_wlist_info_t>& whiteList);
    HcclResult InitHdcWhlistAdd();
    bool IsUsedRdmaLevel0AndIpInvalid();

    HcclResult PrepareHostEnvelopeTemplates(HdcsEmbeddingServiceParam &para);
    HcclResult FillEnvelope(const HdcsEmbeddingServiceParam &para, u32 rankId, HcclEnvelope &envelope);
    HcclResult AsyncCopyHostEnvelopeToDev(const HdcsEmbeddingServiceParam &para, void *stream);
    HcclResult NetWorkRdmaSetQpDepth();
    HcclResult WaitBuildLinkComplete(const u32 remoteRank, const std::unique_ptr<TransportHeterog>& transportPtr);

    template<typename AlgType> std::unique_ptr<AlgBase>& GenerateEmbAlg(u32 tag)
    {
        auto it = tagAlgMap_.Emplace(tag, std::make_unique<AlgType>());

        return it.first->second;
    }

    u32 deviceNumPerServer_;
    HcclDispatcher dispatcher_; // dispatcher放到最后析构
    HcclDispatcher vDispatcher_; // virtualDispatcher放到最后析构
    std::unique_ptr<NotifyPool> notifyPool_;
    std::unique_ptr<QueueNotifyManager> queueNotifyManager_;
    std::unique_ptr<HcclCallbackTask> callbackTask_;
    std::atomic_flag initializedFlag_;
    u32 userRank_;  // 本group中的userrank
    u32 realUserRank_;  // world group中的userrank
    u32 userRankSize_;
    std::vector<RankInfo> rankInfoList_;  // world group内rank的信息, 按照rank id递增依次排列
    bool drvInit_;                          // ra是否初始化
    ServRankInfo_t servRankInfo_;
    std::string serverId_;
    std::unordered_map<u32, std::unordered_map<int, std::vector<int>>> pairLinkInfo_; // server内所有device间的链路类型
    bool inlineReduceSwitchOn_;
    NICDeployment nicDeployment_;
    u32 devicePhyId_;
    s32 deviceLogicId_;
    std::vector<HcclIpAddress> devIpAddr_;
    HcclIpAddress hostIp_;
    u32 hostPort_;
    u32 localRank_;
    SocketHandle hostSocketHandle_;
    SocketHandle hostHeterogSocketHandle_;
    SocketHandle loopbackHeterogSocketHandle_;
    bool isUsedRdmaLevel0_; // 节点内是否使用rdma, 包括a+x和标卡
    bool nicInitialized_;
    bool heterogHostNicInitialized_;
    bool hcomGroupNicInit_;
    // profiling 相关资源
    HcomProfilingMode profilingMode_;
    std::string profilingOption_;
    bool raResourceInit_;
    bool interServer_;
    std::unique_ptr<WorkspaceResource> workSpaceRes_;
    std::vector<u32> enableP2PDevices_;
    bool isSingleMeshAggregation_;
    CCLBufferManager cclBufferManager_;
    bool isExecuteProfilingInit_;
    DevType deviceType_;
    std::string collectiveId_;
    bool heterogRaInit_;
    bool isHeterogComm_;
    bool hostRdmaInitFlag_;
    bool csCommInitFlag_;
    HcclComm commHandle_;
    std::vector<u32> ranksPort_;
    std::vector<u32> groupRanksPort_;
    std::vector<RoleTableNodeInfo> serversInfo_; // roleTable中servers的信息
    std::vector<RoleTableNodeInfo> clientsInfo_; // roleTable中clients的信息
    std::mutex transportHeterogMapMutex_;
    // key: tag, rankId, 将map改成unordered_map改动有点大，计划修改
    UniversalConcurrentMap<s32, std::map<u32, std::unique_ptr<TransportHeterog>>> transportHeterogMap_;

    std::mutex tagOpExecutorMapMutex_;
    UniversalConcurrentMap<s32, std::unique_ptr<OpExecutor>> tagOpExecutorMap_;

    std::map<s32, ServiceHandle> tagServiceHandleMap_;
    std::mutex tagServiceHandleMapMutex_;
    UniversalConcurrentMap<s32, std::unique_ptr<AlgBase>> tagAlgMap_;
    std::unique_ptr<MrManager> mrManager_;
    std::mutex transferMemsMapMutex_;
    std::map<s32, std::vector<DeviceMem>> transferMemsMap_;
    UniversalConcurrentMap<u32, std::map<u32, std::vector<DeviceMem>>> tagRankTransMemsMap_;
    std::unordered_map<std::string, std::map<u32, HcclIpAddress>> rankDevicePhyIdNicInfoMap_;
    DeviceMem csInitDefaultMem_; // 用于embedding service建链时使用
    std::unordered_map<u32, HcclRtContext> rtCtxMap_; // {devPhyId, rtCtx}
    WorkMode commWorkMode_;
    u32 meshAggregationRankSize_;
    std::map<HcomOperationType, std::string> opTypeTagMap_;
    bool isHaveCpuRank_;
    bool isUseRankPort_{ false };
    bool isSetHDCModeInfo_{ false };
    std::map<std::string, HostMem> tagWorkSpaceMem_;
    std::string identifier_;
    u32 ranktableCrc_;
    s32 devicePid_;
    std::unique_ptr<LocklessRingMemoryAllocate<HcclMessageInfo>> pMsgInfosMem_;
    std::unique_ptr<LocklessRingMemoryAllocate<HcclRequestInfo>> pReqInfosMem_;
    std::unique_ptr<HeterogMemBlocksManager> memBlocksManager_;
    std::unique_ptr<LocklessRingMemoryAllocate<RecvWrInfo>> pRecvWrInfosMem_;
    TransportResourceInfo transportResourceInfo_;
    bool multiModuleDiffDeviceNumMode_;
    DeviceMem commContext_;
    std::shared_ptr<ProfilerManager> profilerManager_;
    bool isStandardCard_ = false;
    std::string superPodId_;
    bool isUsedInterHccsMode_ = false;
    std::vector<RankInfo> worldRankInfoList_;

    bool IsAtomicInit();
    bool IsSupportEnableRoce();
    bool IsEnableRoce();

    HcclResult InitWorkSpace();
    void ReleaseWorkSpacebuffer();
    HcclResult CreateWorkSpace(u64 size, DeviceMem &buffer) const;
    HcclResult GetWorkSpace(u64 *workSpaceSize, u64 *workSpace) const;
    HcclResult PrepareCommStart(const std::string &tag, void *addr, u64 count, HcclDataType dataType,
        u32 peerRank, u32& status);
    HcclResult PrepareCommQuerry(const std::string &tag, u32& status);

    void NotifyPrepareComm();
    void WaitPrepareComm();
    void StopPrepareComm();

    HcclResult GetModuleIdx(const RankInfo_t &rankInfo, u32 &moduleIdx);
    HcclResult GetModuleInfo(const std::vector<RankInfo_t> &rankList);
    HcclResult CheckSingleServerComm(const std::vector<RankInfo_t> &rankList) const;
    // HcclImplBase中Comm资源是否存在
    inline bool IsExistCommRes(const std::string &tag)
    {
        std::unique_lock<std::mutex> commLock(commLock_);
        return (tagCommInfo_.find(tag) != tagCommInfo_.end());
    }

    // HcclImplBase中MutiStream资源是否存在
    inline bool IsExistMutiStreamRes(const std::string &tag)
    {
        std::unique_lock<std::mutex> mutiStreamLock(tagStreamInfoLock_);
        return (tagStreamInfo_.find(tag) != tagStreamInfo_.end());
    }

    HcclResult PrepareGather(const std::string &tag, const HcomCollOpInfo &opInfo, HcomOperationType opType);
    HcclResult PrepareBroadcast(const std::string &tag, const HcomCollOpInfo &opInfo, HcomOperationType opType);
    HcclResult GetDeviceNicInfo(RankInfo& rankInfo);
    HcclResult GetDevIP(const HcclIpAddress& hostIp, const u32& devicePhyId,
        HcclIpAddress& ip);
    void GetAndSetSyncMode(SyncMode& preSyncMode, SyncMode newSyncMode);
    void RestorePreSyncMode(SyncMode preSyncMode, SyncMode newSyncMode);

    u32 GetLocalNicPort();
    HcclResult IsLocalDevNicInPlane(bool &isMultipleDeviceNicsUsedByPS);
    HcclResult IsRemoteDevNicInPlane(bool &isRemoteDevNicInPlane);
    HcclResult GetDevIP(const u32 &psRankId, const HcclIpAddress& hostIp, u32 &devicePhyId,
        HcclIpAddress& NicIp);

    HcclResult DeinitAndClearTransportMap();
    HcclResult ClearPsTagNodesInfo();
    bool IsNeedGlobalMrManager();

    HcclIpAddress loopBackIp_;
    bool profilingInitiated_;
    u64 callbackThreadId_;
    u32 role_;
    bool psRaInit_;
    bool workerRaInit_;
    bool mrManagerInit_;
    bool srqInit_;
    std::map<u64, std::vector<rtStream_t>> callbackStreamMap_;
    TransportStorageMap transportStorage_;
    SpinMutex transportMapSpinMutex_;
    bool firstBuildTransportFlag_;
    bool isHostUseDevNic_;
    u32 buildTransportRefCount_;
    std::mutex socketListenMutex_;
    u32 embeddingReduceThreadNum_;

#ifndef HCCD
    TbeReduce::TbeUnsortedSegmentSum aiCoreUnsortedSegmentSum_{};
    TbeReduce::TbeCrackCleard aiCoreCrackCleard_{};
#endif

    HcclAlg *implAlg_ = nullptr;

    HcclWorkflowMode workflowMode_{HcclWorkflowMode::HCCL_WORKFLOW_MODE_OP_BASE};
    bool isCpuRank_;
    u32 deviceNumPerAggregation_;
    std::vector<u32> nicList_;
    std::unordered_map<u32, u32> pairLinkCounter_; // server内所有device间的链路类型计数
    bool isAllRankSamePlane_;
    std::unique_ptr<TopoInfoParse> topoInfoParse_; // 对rank table device选取的校验模块
    u32 serverNum_;
    u32 moduleNum_;
    u32 superPodNum_ = 0;
    bool isAlgoLevel1Default_ = false;
    u32 workSpaceSize_;
    DeviceMem workSpace_;
    bool isDiffDeviceModule_;
    bool threadPrepareCommAsyncExit_;
    std::mutex prepareRequestMutex_;
    std::map<std::string, HcclOpCommDesc>prepareRequest_;
    std::unique_ptr<std::thread> prepareCommThread_;  // 建链所需线程
    std::mutex startPrepareCommMtx_;
    std::condition_variable startPrepareComm_;
    bool isBoardVersion_{ false };
    bool ifBoardVersionInit_{ false };
    tagCommInfo_t tagCommInfo_;    // 以tag为粒度分配comm实例和资源
    std::mutex commLock_;
    tagStreamInfo_t tagStreamInfo_;
    std::mutex tagStreamInfoLock_;

    // 区分不同的tag
    std::unordered_map<u32, std::unique_ptr<HcclEsRdmaInfoForUpdate[]>> rdmaEnvelopeInfoHostMems_{};
    std::vector<Stream> auxRingCommStreamsDev_;
    std::unique_ptr<HcclTraceInfo> opBaseAtraceInfo_; // opbase层关键日志
    bool isServerInter_{ false };
    bool isSupportRdmaLite_{ false };       // 是否支持rdma lite

    std::unique_ptr<HcclSocketManager> socketManager_;
    HcclIpAddress localVnicIp_;
    u32 localVnicListenPort_;
    std::map<HcclIpAddress, HcclNetDevCtx> netDevCtxMap_;
    bool retryEnable_ = false;
    std::unordered_map<std::string, void *> esWorkMems_{};
    std::mutex tagNodesInfoLock_;
    std::unordered_map<u32, std::vector<RoleTableNodeInfo>> tagNodesInfo_{};
    bool isEsAllLinkModel_ = false;
};

}  // end namespace hccl
#endif  // HCCL_IMPL_BASE_H
